﻿#region 命名空间

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Sharp.Domain.CMSBoundedContext.AritcleModule.Aggregates;
using Sharp.Infrastructure;
using Sharp.Infrastructure.Crosscutting;
using Sharp.Infrastructure.Network;

#endregion

namespace Sharp.Domain.CMSBoundedContext.AritcleModule.GatherEngine
{
    /// <summary>
    ///     采集内容页
    /// </summary>
    public class GatherContentPage : IGatherContentPage
    {
        #region 获取文章内容页的下页url

        /// <summary>
        ///     获取文章内容页的下页url
        /// </summary>
        /// <param name="rulesList"></param>
        /// <param name="doc"></param>
        /// <param name="strWebCode"></param>
        /// <returns></returns>
        private string GetArticleNextPageURL(IEnumerable<Gather> rulesList, HtmlDocument doc, string strWebCode)
        {
            var articleNextPage = "";
            foreach (var ag in rulesList.Where(m => m.ContentPageNextUrl.IsNullOrEmpty() == false))
            {
                switch (GatherRule.Analysis(ag.ContentPageNextUrl))
                {
                    case GatherRuleType.XPath:
                        var hn = doc.DocumentNode.SelectSingleNode(ag.ContentPageNextUrl.Substring(2));
                        articleNextPage = hn == null ? string.Empty : hn.Attributes["href"].Value;
                        break;
                    case GatherRuleType.DomQuery:
                        hn = doc.SingleGet(ag.ContentPageNextUrl.Substring(2));
                        articleNextPage = hn == null ? string.Empty : hn.Attributes["href"].Value;
                        break;
                    default:
                        articleNextPage = Utilities.GetInfo(strWebCode, ag.ContentPageNextUrl);
                        break;
                }
                if (string.IsNullOrEmpty(articleNextPage) == false)
                {
                    break;
                }
            }
            if (articleNextPage.EndsWith("#") ||
                articleNextPage.StartsWith("javascript", StringComparison.CurrentCultureIgnoreCase))
            {
                return string.Empty;
            }
            return articleNextPage;
        }

        #endregion

        #region 获取文章内容页的内容

        /// <summary>
        ///     获取文章内容页的内容
        /// </summary>
        /// <param name="rulesList"></param>
        /// <param name="doc"></param>
        /// <param name="strWebCode"></param>
        /// <returns></returns>
        private string GetArticleContent(IList<Gather> rulesList, HtmlDocument doc, string strWebCode)
        {
            foreach (var ag in rulesList.Where(m => m.Content.IsNullOrEmpty() == false))
            {
                string strContent;
                switch (GatherRule.Analysis(ag.Content))
                {
                    case GatherRuleType.XPath:
                        var _hn = doc.DocumentNode.SelectSingleNode(ag.Content.Substring(2));
                        strContent = _hn == null ? null : _hn.InnerHtml;
                        break;
                    case GatherRuleType.DomQuery:
                        strContent = doc.SingleGetInnerHtml(ag.Content.Substring(2));
                        break;
                    default:
                        strContent = Utilities.GetInfo(strWebCode, ag.Content);
                        break;
                }

                if (string.IsNullOrWhiteSpace(strContent) == false) //如果采集到的就跳出循环
                {
                    return strContent.Trim();
                }
            }
            return "";
        }

        #endregion

        #region 是否为忽略网址

        /// <summary>
        /// </summary>
        /// <param name="rulesList"></param>
        /// <param name="url"></param>
        /// <returns></returns>
        public bool IsIgnoreUrl(IList<Gather> rulesList, string url)
        {
            foreach (var ag in rulesList.Where(m => string.IsNullOrEmpty(m.IgnoreUrl) == false))
            {
                var reg = new Regex(ag.IgnoreUrl, RegexOptions.IgnoreCase);
                if (reg.IsMatch(url))
                {
                    return true;
                }
            }
            return false;
        }

        #endregion

        #region - 变量 -

        private static IArticleRepository _articleRepository;

        private static IGatherRepository _gatherRepository;

        private static IGatherHistoryRepository _historyRepository;

        private IList<Gather> _rulesList;

        private Gather _gatherModel;

        #endregion

        #region - 属性 -

        public string PicPath { get; set; }

        private WebParameter _wp;

        #endregion

        #region - 构造函数 -
        public GatherContentPage(IArticleRepository articleRepository, IGatherRepository gatherRepository, IGatherHistoryRepository historyRepository)
        {
            _articleRepository = articleRepository;
            _gatherRepository = gatherRepository;
            _historyRepository = historyRepository;
        }
        /// <summary>
        /// 初始化
        /// </summary>
        /// <param name="wp"></param>
        public void Init(WebParameter wp)
        {
            this._wp = wp;
            this._rulesList = _gatherRepository.GetRules(this._wp.GatherId.ToString());
            this._gatherModel = _gatherRepository.Get(this._wp.GatherId);

            #region 校验基本条件

            PicPath = "";
            if (string.IsNullOrEmpty(PicPath))
            {
                PicPath = "/UpLoadFolder/";
            }

            try
            {
                var di = new DirectoryInfo(WebServer.MapPath(PicPath));
                if (!di.Exists)
                {
                    di.Create();
                }
            }
            catch
            {
            }

            #endregion
        }

        public void Init(Guid gatherId)
        {
            this._wp = new WebParameter()
            {
                GatherId = gatherId
            };
            if (_rulesList == null)
                this._rulesList = _gatherRepository.GetRules(gatherId.ToString());
            if (_gatherModel == null)
                this._gatherModel = _gatherRepository.Get(gatherId);
        }

        #endregion

        #region 采集内容页

        /// <summary>
        ///     采集内容页
        /// </summary>
        /// <param name="Url"></param>
        /// <returns></returns>
        public ContentPageGatherResult GatherPage(object Url)
        {
            try
            {
                var url = new Uri(Url.ToString());
                return GatherPage(url, false);
            }
            catch (Exception ex)
            {
                LogHelper.Error(ex);

                var cpgr = new ContentPageGatherResult
                {
                    URL = Url.ToString(),
                    Messages = ex.Message,
                    Gathered = false,
                    Type = ContentPageGatherResultType.Alert
                };

                return cpgr;
            }
        }

        /// <summary>
        ///     采集内容页
        /// </summary>
        /// <param name="url"></param>
        /// <param name="fullArticleScaned">已经扫描过全文文章地址</param>
        private ContentPageGatherResult GatherPage(Uri url, bool fullArticleScaned)
        {
            //LogHelper.Verbose("this._wp.GatherId.ToString()：" + gather_model.ID + "\r\nUrl：" + _Url.ToString());

            ContentPageGatherResult cpgr;
            if (IsIgnoreUrl(this._rulesList, url.ToString()))
            {
                cpgr = new ContentPageGatherResult
                {
                    URL = url.ToString(),
                    Messages = "忽略采集的网址。",
                    Gathered = false,
                    Type = ContentPageGatherResultType.Right
                };
                return cpgr;
            }

            cpgr = new ContentPageGatherResult();
            var articleShowUrl = url.ToString();
            cpgr.URL = articleShowUrl;

            bool urlGathered;
            lock (_gatherRepository)
            {
                urlGathered = _historyRepository.Exists(articleShowUrl);
                if (urlGathered)
                {
                    cpgr.Messages = "url已经采集过。";
                    goto Exit;
                }
            }

            if (this._wp.IsUpdate || urlGathered == false)
            {
                var engine = new NetHelper
                {
                    EncodingString = this._rulesList.First(x => x.ParentId == null).Encoding
                };
                var info = engine.Get(url);

                if (info.StatusCode == HttpStatusCode.NotFound)
                {
                    cpgr.Type = ContentPageGatherResultType.Alert;
                    cpgr.Messages = "访问远程资源发生一般性错误。" + info.Code;
                    goto Exit;
                }
                if (info.StatusCode != HttpStatusCode.OK)
                {
                    cpgr.Type = ContentPageGatherResultType.Err;
                    cpgr.Messages = "访问远程资源错误。" + info.Code;
                    goto Exit;
                }
                if (info.Html.Length < 2048)
                {
                    cpgr.Type = ContentPageGatherResultType.Alert;
                    cpgr.Messages = "采集到网页过小。";
                    goto Exit;
                }
                cpgr.Charset = info.Charset;

                return DecomposePage(url.ToString(), fullArticleScaned, info.Html);
            }

            Exit:
            cpgr.Gathered = false;
            cpgr.Messages += cpgr.Charset;

            return cpgr;
        }

        #endregion

        /// <summary>
        /// 拆解网页
        /// </summary>
        /// <returns></returns>
        public ContentPageGatherResult DecomposePage(string strUrl, bool fullArticleScaned, string webhtml)
        {
            Uri url = new Uri(strUrl);

            var gathered = false;

            var cpgr = new ContentPageGatherResult();
            cpgr.GatherId = this._wp.GatherId;
            try
            {
                var articleShowUrl = url.AbsoluteUri;
                cpgr.URL = articleShowUrl;



                var downImg = this._gatherModel.CanDownImg;
                var drawWaterMark = this._gatherModel.CanDrawWaterMark;

                bool urlGathered;
                lock (_gatherRepository)
                {
                    urlGathered = _historyRepository.Exists(x => x.Url == articleShowUrl);
                }

                if ((this._wp != null && this._wp.IsUpdate) || urlGathered == false)
                {
                    #region 采集文章

                    IList<string> contentList = new List<string>();
                    var model = new Article
                    {
                        GatherId = _wp.GatherId
                    };
                    if (url.Host != "cache.baiducontent.com")
                    {
                        model.Url = url.ToString();
                    }


                    var doc = new HtmlDocument();
                    //载入HTML
                    doc.LoadHtml(webhtml);

                    #region 获取内容标题

                    foreach (var ag in this._rulesList.Where(m => m.ContentTitle.IsNullOrEmpty() == false))
                    {
                        switch (GatherRule.Analysis(ag.ContentTitle))
                        {
                            case GatherRuleType.XPath:
                                var hn = doc.DocumentNode.SelectSingleNode(ag.ContentTitle.Substring(2));
                                model.Title = hn == null ? null : hn.InnerText.Trim();
                                break;
                            case GatherRuleType.DomQuery:
                                model.Title = doc.SingleGetInnerText(ag.ContentTitle.Substring(2));
                                break;
                            default:
                                model.Title = Utilities.GetInfo(webhtml, ag.ContentTitle);
                                break;
                        }

                        if (string.IsNullOrEmpty(model.Title) == false)
                        {
                            break;
                        }
                    }
                    if (string.IsNullOrEmpty(model.Title))
                    {
                        cpgr.Type = ContentPageGatherResultType.Err;
                        cpgr.Messages = "采集不到标题。";
                        goto Exit;
                    }

                    #endregion

                    Guid existsArticleId; //已经存在的文章的ID
                    lock (_gatherRepository)
                    {
                        if (_articleRepository.Exists(model.Title, out existsArticleId)) //查找相同的文章
                        {
                            _historyRepository.LogUrl(articleShowUrl);

                            if (this._wp.IsUpdate == false)
                            {
                                cpgr.Type = ContentPageGatherResultType.Right;
                                cpgr.Messages = $"“{model.Title}”文章标题已经存在。";
                                goto Exit;
                            }
                        }
                    }

                    #region 获取全文地址

                    if (fullArticleScaned == false)
                    {
                        foreach (var ag in this._rulesList.Where(m => string.IsNullOrEmpty(m.FullArticleUrl) == false))
                        {
                            string fullArticleUrl;
                            switch (GatherRule.Analysis(ag.FullArticleUrl))
                            {
                                case GatherRuleType.XPath:
                                    var hn = doc.DocumentNode.SelectSingleNode(ag.FullArticleUrl.Substring(2));
                                    fullArticleUrl = hn == null ? null : hn.InnerText;
                                    break;
                                case GatherRuleType.DomQuery:
                                    hn = doc.SingleGet(ag.FullArticleUrl.Substring(2));
                                    fullArticleUrl = hn == null ? string.Empty : hn.Attributes["href"].Value;
                                    break;
                                default:
                                    fullArticleUrl = Utilities.GetInfo(webhtml, ag.FullArticleUrl);
                                    break;
                            }

                            //如果发现“全页”链接，重头采集
                            if (string.IsNullOrEmpty(fullArticleUrl) == false)
                            {
                                fullArticleUrl = Utilities.FormatUrl(articleShowUrl, fullArticleUrl);
                                return GatherPage(new Uri(fullArticleUrl), true);
                            }
                        }
                    }

                    #endregion

                    var isScan = false;
                    //文章来源
                    foreach (var ag in this._rulesList.Where(m => m.ComeFrom.IsNullOrEmpty() == false))
                    {
                        isScan = true;
                        switch (GatherRule.Analysis(ag.ComeFrom))
                        {
                            case GatherRuleType.XPath:
                                var hn = doc.DocumentNode.SelectSingleNode(ag.ComeFrom.Substring(2));
                                model.ComeFrom = hn?.InnerText;
                                break;
                            case GatherRuleType.DomQuery:
                                model.ComeFrom = doc.SingleGetInnerText(ag.ComeFrom.Substring(2));
                                break;
                            default:
                                model.ComeFrom = Utilities.GetInfo(webhtml, ag.ComeFrom);
                                break;
                        }

                        if (string.IsNullOrEmpty(model.ComeFrom) == false)
                        {
                            break;
                        }
                    }
                    if (string.IsNullOrEmpty(model.ComeFrom) && isScan)
                    {
                        cpgr.Type = ContentPageGatherResultType.Alert;
                        cpgr.Messages = "采集不到文章来源。";
                    }
                    isScan = false;
                    //文章作者
                    foreach (var ag in this._rulesList.Where(m => m.Author.IsNullOrEmpty() == false))
                    {
                        isScan = true;
                        switch (GatherRule.Analysis(ag.Author))
                        {
                            case GatherRuleType.XPath:
                                var hn = doc.DocumentNode.SelectSingleNode(ag.Author.Substring(2));
                                model.Author = hn?.InnerText;
                                break;
                            case GatherRuleType.DomQuery:
                                model.Author = doc.SingleGetInnerText(ag.Author.Substring(2));
                                break;
                            default:
                                model.Author = Utilities.GetInfo(webhtml, ag.Author);
                                break;
                        }

                        if (string.IsNullOrEmpty(model.Author) == false)
                        {
                            break;
                        }
                    }
                    if (string.IsNullOrEmpty(model.Author) && isScan)
                    {
                        cpgr.Type = ContentPageGatherResultType.Alert;
                        cpgr.Messages = "采集不到文章作者。";
                        isScan = false;
                    }

                    #region 采集文章更新时间

                    foreach (var ag in this._rulesList.Where(m => m.UpdateTime.IsNullOrEmpty() == false))
                    {
                        string strdatatime;
                        switch (GatherRule.Analysis(ag.UpdateTime))
                        {
                            case GatherRuleType.XPath:
                                var hn = doc.DocumentNode.SelectSingleNode(ag.UpdateTime.Substring(2));
                                strdatatime = hn == null ? null : hn.InnerText;
                                break;
                            case GatherRuleType.DomQuery:
                                strdatatime = doc.SingleGetInnerText(ag.UpdateTime.Substring(2));
                                break;
                            default:
                                strdatatime = Utilities.GetInfo(webhtml, ag.UpdateTime);
                                break;
                        }

                        if (string.IsNullOrEmpty(strdatatime))
                        {
                            continue; //采集不到就继续下一个规则
                        }
                        strdatatime = strdatatime.Trim();
                        DateTime dt;
                        if (DateTime.TryParse(strdatatime, out dt))
                        {
                            var s = dt.ToString("HH:mm:ss.ffff");
                            if (s == "00:00:00.0000")
                            {
                                var e = DateTime.Now.ToString(" HH:mm:ss.ffff");
                                dt = DateTime.Parse(string.Concat(dt.ToString("yyyy-MM-dd"), e));
                            }
                            s = dt.ToString("ss.ffff");
                            if (s == "00.0000")
                            {
                                var e = DateTime.Now.ToString("ss.ffff");
                                dt = DateTime.Parse(string.Concat(dt.ToString("yyyy-MM-dd HH:mm:"), e));
                            }
                            model.EventDay = dt;
                            break;
                        }
                    }
                    if (model.EventDay == null)
                    {
                        cpgr.Type = ContentPageGatherResultType.Alert;
                        cpgr.Messages = "采集不到文章更新时间。";
                    }

                    #endregion

                    var strContent = GetArticleContent(this._rulesList, doc, webhtml);

                    if (string.IsNullOrEmpty(strContent))
                    {
                        cpgr.Type = ContentPageGatherResultType.Err;
                        cpgr.Messages = "文章内容采集不到。";
                        goto Exit;
                    }


                    contentList.Add(strContent);

                    NetHelper engine;
                    //采集文章页内的下页
                    if (this._gatherModel.ContentPageNextUrl.IsNullOrEmpty() == false && fullArticleScaned == false)
                    {
                        //文章页下一页
                        var articleNextPage = GetArticleNextPageURL(this._rulesList, doc, webhtml);

                        if (articleNextPage.IsNullOrEmpty() == false &&
                            !Utilities.FormatUrl(articleShowUrl, articleNextPage)
                                      .Equals(url.ToString(), StringComparison.CurrentCultureIgnoreCase))
                        {
                            articleNextPage = Utilities.FormatUrl(articleShowUrl, articleNextPage);
                            do
                            {
                                engine = new NetHelper();

                                var info = engine.Get(articleNextPage, articleShowUrl);

                                if (string.IsNullOrEmpty(info.Code) == false)
                                {
                                    cpgr.Type = ContentPageGatherResultType.Err;
                                    cpgr.Messages = "访问“下一页”出错。" + info.Code;
                                    goto Exit;
                                }
                                webhtml = info.Html;

                                //载入HTML
                                doc = new HtmlDocument();
                                doc.LoadHtml(info.Html);

                                strContent = GetArticleContent(this._rulesList, doc, webhtml);

                                if (string.IsNullOrEmpty(strContent)) //如果都循环完了还是采集不到
                                {
                                    cpgr.Type = ContentPageGatherResultType.Alert;
                                    cpgr.Messages = "采集不到“下一页”(" + articleNextPage + ")文章内容。";
                                }

                                #region 检测是否死循环

                                if (contentList.Contains(strContent))
                                {
                                    cpgr.Type = ContentPageGatherResultType.Err;
                                    cpgr.Messages = "出现疑似死循环，请检查采集规则。";
                                    goto Exit;
                                }

                                #endregion

                                contentList.Add(strContent);

                                var tempArticleNextPage = GetArticleNextPageURL(this._rulesList, doc, webhtml);
                                if (string.IsNullOrWhiteSpace(tempArticleNextPage))
                                {
                                    break;
                                }

                                tempArticleNextPage = Utilities.FormatUrl(articleShowUrl, tempArticleNextPage);

                                if (articleNextPage.Equals(url.ToString(), StringComparison.CurrentCultureIgnoreCase)
                                    ||
                                    articleNextPage.Equals(tempArticleNextPage,
                                        StringComparison.CurrentCultureIgnoreCase)
                                    ) //防止最后一页的“下一页“就是本身
                                {
                                    articleNextPage = string.Empty;
                                }
                                else
                                {
                                    articleNextPage = tempArticleNextPage;
                                }
                            } while (articleNextPage.IsNullOrEmpty() == false);
                        }
                    }

                    //过滤
                    model.Content = string.Join("@Next@", contentList.ToArray());
                    model.Content = Filter.GatherFilter(model.Content, this._gatherModel.Filter,
                        this._gatherModel.Replace);
                    model.Content = Utilities.FormatUrlInHtml(articleShowUrl, model.Content, this._wp.IsLeech);
                    model.Content = Filter.FilterUrl(model.Content); //去除多余的网页中的链接
                    model.Content = HTML.NoHtml(model.Content,
                        WebSourceTag.Note | WebSourceTag.Invisible | WebSourceTag.Script | WebSourceTag.Style);


                    var yes = true; //是否采集文章。


                    var picurLs = Utilities.Get_PicUrl(model.Content, url.ToString()).ToList();
                    if (picurLs.Count > 0 && this._wp.IsTest == false)
                    {
                        //如果要下载图片而且要采集有图片的文章
                        if (downImg)
                        {
                            if (this._wp.IsTest == false && downImg)
                            {
                                //是允许采集
                                engine = new NetHelper();
                                var allowDownPic = engine.UrlExist(picurLs, this._wp.BadLinkNum);

                                #region 采集图片和加水印

                                if (allowDownPic)
                                {
                                    var myWebClient = new WebClient();
                                    //var drawingImage = new DrawingWaterMarkImage();
                                    foreach (var getPicUrl in picurLs)
                                    {
                                        var name = Path.GetFileName(getPicUrl);

                                        //下载图片
                                        try
                                        {
                                            myWebClient.DownloadFile(getPicUrl,
                                                WebServer.MapPath(PicPath + "\\" + name));
                                        }
                                        catch (Exception ex)
                                        {
                                            LogHelper.Error(ex);
                                            yes = false;
                                            cpgr.Type = ContentPageGatherResultType.Err;
                                            cpgr.Messages = (ex.ToString());
                                        }
                                        finally //这个图片如果采集成功就替换成本地链接
                                        {
                                            if (drawWaterMark && yes)
                                            {
                                                //drawingImage.WatherMark(WebServer.MapPath(PicPath + "\\" + name));
                                                //加盖水印
                                            }
                                            model.Content = model.Content.Replace(getPicUrl, PicPath + name);
                                        }
                                    }
                                }
                                else
                                {
                                    cpgr.Messages = ("文章含有图片死链接，丢弃它。");
                                    yes = false;
                                }

                                #endregion
                            }
                        }
                        else
                        {
                            yes = true;
                        }
                    }

                    if (yes && !this._wp.IsTest)
                    {
                        Result res = new Result();

                        #region 使用默认缩略图

                        if (picurLs.Count > 0)
                        {
                            model.IncludePic = true;
                            model.PicUrl = picurLs[0];
                        }

                        #endregion

                        if (this._wp.IsUpdate == false)
                        {
                            #region 添加文章

                            model.Id = Guid.Empty;
                            model.CheckInTime = DateTime.Now;
                            _articleRepository.Add(model);

                            #endregion
                        }
                        else
                        {
                            #region 修改文章

                            if (existsArticleId == Guid.Empty) goto Exit;

                            model.Id = existsArticleId;
                            _articleRepository.Update(model);

                            #endregion
                        }
                        if (string.IsNullOrEmpty(res.Err))
                        {
                            lock (_gatherRepository)
                            {
                                _historyRepository.LogUrl(articleShowUrl);
                            }
                            cpgr.Messages = ("文章采集成功");
                            cpgr.Type = ContentPageGatherResultType.Right;
                            gathered = true;
                        }
                        else
                        {
                            cpgr.Type = ContentPageGatherResultType.Err;
                            cpgr.Messages = res.Err;
                        }
                    }
                    else
                    {
                        if (this._wp.IsTest)
                        {
                            gathered = true;
                            cpgr.Messages = "文章采集成功。(测试采集未入库)";
                        }
                        else
                        {
                            cpgr.Messages = "自动跳过采集。";
                        }
                    }

                    #endregion
                }
                else
                {
                    cpgr.Messages = "已经采集过。";
                    gathered = false;
                }
            }
            catch (Exception ex)
            {
                LogHelper.Error(ex);
                cpgr.Type = ContentPageGatherResultType.Err;
                cpgr.Messages = ex.Message;
            }
            Exit:
            if (gathered)
            {
                _gatherRepository.LastWork(this._wp.GatherId);
            }
            cpgr.Gathered = gathered;
            cpgr.Messages += cpgr.Charset;

            return cpgr;
        }
    }
}